# -*- coding: utf-8 -*-
import requests
from datetime import datetime
from bs4 import BeautifulSoup
from food.models import Food, Page, Market, Price, Err_price


# payload = {'id':'650'}
# r = requests.get("http://www.tcprice.gov.cn/news1.asp", params=payload)
# r.encoding = 'gbk'
# # print r.text
# soup = BeautifulSoup(r.text,'html5lib')
# tbody = soup.find('td',{'class','ask3_infolist'}).find('tbody')
# rows = tbody.find_all('tr')




# 市场 一共13个
# markets = rows[2].find_all('p')
# num = len(markets)
# print num
# j = 0
# for m in markets:
#     name = m.get_text()
#     print name
#     if j < num-1:
#         market_model = Market.objects.create_market(market_name=name)
#     j += 1
a = datetime.now()
pages = Page.objects.all().order_by('page_date')
markets = Market.objects.all().order_by('id')
foods = Food.objects.all().order_by('id')
fail_page_id = []
for p in pages:

    id = p.page_id
    payload = {'id':id}
    r = requests.get("http://www.tcprice.gov.cn/news1.asp", params=payload)
    r.encoding = 'gbk'
    # print r.text
    soup = BeautifulSoup(r.text,'html5lib')
    tbody = soup.find('td',{'class','ask3_infolist'}).find('tbody')
    if tbody is not None:
        rows = tbody.find_all('tr')
        # 副食品种类 一共121种
        i = 0
        type = len(rows)
        if type == 125:

            # print foods
            page = p
            # print markets
            for row in rows:
                cols = row.find_all('td')
                cols_num = len(cols)

                food = "".join(cols[0].text.strip().split())

                if 2 < i < type-1:
                    # print i
                    # print foods[i-3]

                    j = 0
                    prices = []
                    Err_prices = []
                    for c in cols:
                        if 0 < j < cols_num-1:
                            price_str = c.get_text().strip()
                            if price_str:
                                try:
                                    price = float(price_str)

                                    m = markets[j-1]
                                    price_model = Price(price=price, page=page, food=foods[i-3], market=markets[j-1])
                                    prices.append(price_model)
                                except:
                                    price = price_str
                                    print price
                                    err_price_model = Err_price(price=price, page=page, food=foods[i-3], market=markets[j-1])
                                    Err_prices.append(err_price_model)
                            else:
                                price = None
                                m = markets[j-1]
                                price_model = Price(price=price, page=page, food=foods[i-3], market=markets[j-1])
                                prices.append(price_model)


                        j += 1
                    Price.objects.bulk_create(prices)
                    Err_price.objects.bulk_create(Err_prices)

                i += 1
            print 1,id

        else:
            fail_page_id.append(id)
            print 0,id
    else:
        fail_page_id.append(id)
        print 0,id
        p.page_ok = False
        p.save()


b = datetime.now()
c = b - a
print c
print len(fail_page_id)


